package utils;

import java.io.FileNotFoundException;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;

import utils.*;

import java.io.IOException;  
import java.io.StringReader;   
import org.wltea.analyzer.core.IKSegmenter;  
import org.wltea.analyzer.core.Lexeme; 

public class Nlp {
	
	/**
	 * 常用停用词
	 */
	private static Set<String> stopWordsSet =new HashSet<String>();
	static {
		stopWordsSet.add("的/ude1");
		stopWordsSet.add("我们/rr");
		stopWordsSet.add("我/rr");
		stopWordsSet.add("要");
		stopWordsSet.add("自己/rr");
		stopWordsSet.add("之/uzhi");
		stopWordsSet.add("将/d");
		stopWordsSet.add("后/f");
		stopWordsSet.add("应");
		stopWordsSet.add("到");
		stopWordsSet.add("某");
		stopWordsSet.add("某");
		stopWordsSet.add("后");
		stopWordsSet.add("个");
		stopWordsSet.add("是");
		stopWordsSet.add("位");
		stopWordsSet.add("新");
		stopWordsSet.add("一");
		stopWordsSet.add("两");
		stopWordsSet.add("中");
		stopWordsSet.add("或");
		stopWordsSet.add("有");
		stopWordsSet.add("更");
		stopWordsSet.add("好");
		stopWordsSet.add("");
		stopWordsSet.add(",/wd");
		stopWordsSet.add("“");
		stopWordsSet.add("”");
		stopWordsSet.add("。");
		stopWordsSet.add(".");
		stopWordsSet.add("以便");
		//stopWordsSet.add("8月/t");
		//stopWordsSet.add("人民币/n");
	}
		
	
	//cut words. return a String[].
	public static String[] cutWords(String path) throws FileNotFoundException, IOException{
		//path为文件路径，不是文件夹路径
		System.out.println(path);
		String ss[]=null;
		String sInput=ReadFile.readFile(path);
		String nativeBytes = "";
        StringReader sr=new StringReader(sInput);  
        IKSegmenter ik=new IKSegmenter(sr, true);  
        Lexeme lex=null;  
        while((lex=ik.next())!=null){  
//            System.out.print(lex.getLexemeText()+"|");  
            nativeBytes=nativeBytes+lex.getLexemeText()+" ";
        }
        if(nativeBytes.length()<1){
        	nativeBytes="";
        }
		ss=ReadFile.txtSplit(nativeBytes);
		return ss;
	}
	
	//检查单词是否为停用词
	public static boolean isStopWord(String word)
	{
		return stopWordsSet.contains(word);
	}
	
	//drop the stopwords ,return a String[]
	public static String[] dropStopWords(String[] ss){
		
		List<String> list=new ArrayList<String>();
		int j=0;
		for(int i=0;i<ss.length;i++){
			if(!Nlp.isStopWord(ss[i])){
				//不是停用词
				String temp=ss[i];
				list.add(temp);
			}
		}
		String[] sstop =new String[list.size()];
		sstop=list.toArray(sstop);
		return sstop;
	}
	
	//
	public static String[] strUnique(String[] str){
		List<String> strlist=new ArrayList<String>();
		for(String word:str){
			if(!strlist.contains(word)){
				strlist.add(word);
			}
		}
		final int size=strlist.size();
		String[] arr=(String[])strlist.toArray(new String[size]);
		return arr;
	}
	
//**********//	
}
